# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
from followUserSpider.items import *
from followUserSpider.settings import EXECUTE_QUEUE,DONE_QUEQE,INIT_QUEUE,FANS_QUEUE
from followUserSpider.redisUtils import OPRedis

class FollowUserSpiderPipeline(object):

    def __init__(self):
        self.opr = OPRedis()

    def process_item(self, item, spider):

        if isinstance(item, FollowUserItem):
            userId = item['userId']
            #遍历每个followUserids,判断每个userid是否存在于EXECUTE_QUEQUE 以及 DONE_QUEQE 若没有，则存入 INIT_QUEUE
            for followid in item['followUserids']:
                #存入爬取的关系
                self.opr.sadd(FANS_QUEUE.format(uid=followid),userId)
                #解析item也会并发进行,将新获取的userid放入 任务发配队列前先做一层去重判断
                if(self.opr.sismember(EXECUTE_QUEUE,followid)==False & self.opr.sismember(DONE_QUEQE,followid)==False):
                    #处理解析的值
                    self.opr.lpush(INIT_QUEUE,followid)
        if isinstance(item, FinishUserItem):
            userid = item['userId']
            #把这个id从任务进行中队列移入到任务完成队列
            self.opr.smove(EXECUTE_QUEUE,DONE_QUEQE,userid)
        return item
